package com.java.webMagic;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;

import org.json.JSONException;
import org.json.JSONObject;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;

public class Test implements PageProcessor {

	private static String username = "qq598535550";// 设置csdn用户名
	private static int size;// 共抓取到的文章数量
	private static int num = 1;
	private static OutputStreamWriter outFile;

	// 抓取网站的相关配置，包括：编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);



	    
	    
	    
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		try {
			JSONObject json = new JSONObject(page.getJson().get());
			if (json.get("a").equals("")) {
				System.out.println("该题目无答案");
			} else {
				outFile("题目" + num + "：" + json.get("question") + "\n");
				System.out.println("题目" + num + "：" + json.get("question"));
				if (!json.get("d").equals("")) {
					for (char index = 97; index < 101; index++) {
						outFile(String.valueOf(index) + "、" + json.get(String.valueOf(index)) + "\n");
						System.out.println(String.valueOf(index) + "、" + json.get(String.valueOf(index)));
					}
				}
				int answer = Integer.parseInt(json.get("ta").toString()) + 96;
				outFile("答案：" + (char)answer + "\n");
				System.out.println("答案：" + (char)answer);
				outFile("\n");
				num++;
			}
		} catch (JSONException e) {
			e.printStackTrace();
		}
	}
	public static String createOutFile(String string) {
		try {
		    outFile = new OutputStreamWriter(new FileOutputStream(new File(string)), "UTF-8");
		    // 构建OutputStreamWriter对象,参数可以指定编码,默认为操作系统默认编码,windows上是gbk
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return string + "创建成功!";
	}
	public static String closeOutFile(String string) {
		try {
			outFile.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return string + "关闭成功!";
	}
	public static String outFile(String string) {
		try {
			outFile.write(string);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return string + "写入文件成功";
	}

	public static void main(String[] args) {
		String outFileName = "/Users/yaozhixin/Downloads/科目二.txt";
		String outFileName1 = "/Users/yaozhixin/Downloads/null.urls.txt";
		System.out.println(Test.createOutFile(outFileName));
		long startTime, endTime;
		startTime = System.currentTimeMillis();
		// 从用户博客首页开始抓，开启5个线程，启动爬虫
		
		/*for (size = 1; size < 1300; size++) {
			//Spider.create(new Test()).addUrl("http://mnks.jxedt.com/get_question?index=" + size).thread(1).run();
			Test.outFile("http://mnks.jxedt.com/get_question?index=" + size + "\n");
		}*/
		//Spider.create(new Test()).addUrl("http://mnks.jxedt.com/get_question?index=" + "1").thread(1).run();
		Spider.create(new Test())
		.scheduler(new FileCacheQueueScheduler("/Users/yaozhixin/Downloads"))
		.pipeline(new FilePipeline())
		.thread(1000).run();
		endTime = System.currentTimeMillis();
		System.out.println("【爬虫结束】共抓取" + 1300 + "篇文章，耗时约" + ((endTime - startTime) / 1000) + "秒，已保存到数据库，请查收！");
		System.out.println(Test.closeOutFile(outFileName));
	}

	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
}